{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dog VS Cat - Kaggle Competition\n",
    "\n",
    "- 进入Top20 要求 Loss < 0.04183\n",
    "- 进入Top10 要求 Loss < 0.03807"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import cv2\n",
    "import h5py\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from time import time\n",
    "from datetime import datetime\n",
    "from tqdm import tqdm\n",
    "from utils import get_params_count\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "from keras.applications import inception_v3, xception, resnet50, vgg16, vgg19\n",
    "from keras.applications import InceptionV3, Xception, ResNet50, VGG16, VGG19\n",
    "from keras.layers import Input, Dense, Dropout, Activation, Flatten, Lambda\n",
    "from keras.layers.noise import GaussianDropout\n",
    "from keras.layers.pooling import GlobalAveragePooling2D\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard\n",
    "from keras.models import Model\n",
    "from keras.optimizers import SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████| 12500/12500 [00:26<00:00, 472.32it/s]\n",
      "100%|█████████████████████████████████████████████████████| 12500/12500 [00:27<00:00, 462.04it/s]\n",
      "100%|█████████████████████████████████████████████████████| 12500/12500 [00:26<00:00, 469.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Data Size = 6.24 GB\n",
      "Testing Data Size = 3.12 GB\n"
     ]
    }
   ],
   "source": [
    "height = 299\n",
    "labels = np.array([0] * 12500 + [1] * 12500)\n",
    "train = np.zeros((25000, height, height, 3), dtype=np.uint8)\n",
    "test = np.zeros((12500, height, height, 3), dtype=np.uint8)\n",
    "\n",
    "for i in tqdm(range(12500)):\n",
    "    img = cv2.imread('./train/cat/%s.jpg' % str(i))\n",
    "    img = cv2.resize(img, (height, height))\n",
    "    train[i] = img[:, :, ::-1]\n",
    "    \n",
    "for i in tqdm(range(12500)):\n",
    "    img = cv2.imread('./train/dog/%s.jpg' % str(i))\n",
    "    img = cv2.resize(img, (height, height))\n",
    "    train[i + 12500] = img[:, :, ::-1]\n",
    "\n",
    "for i in tqdm(range(12500)):\n",
    "    img = cv2.imread('./test/%s.jpg' % str(i + 1))\n",
    "    img = cv2.resize(img, (height, height))\n",
    "    test[i] = img[:, :, ::-1]\n",
    "    \n",
    "print('Training Data Size = %.2f GB' % (sys.getsizeof(train)/1024**3))\n",
    "print('Testing Data Size = %.2f GB' % (sys.getsizeof(test)/1024**3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0. CONV + GAP + Dropout + Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train, labels, shuffle=True, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def finetune(MODEL, preprocess, height, freeze_till, lr, nb_epoch, weights=None):\n",
    "    # Preprocess: Standardization\n",
    "    x = Input(shape=(height, height, 3))\n",
    "    x = Lambda(preprocess)(x)\n",
    "\n",
    "    # Base Model: Freeze all conv layers\n",
    "    base_model = MODEL(include_top=False, input_tensor=x, weights='imagenet', pooling='avg')\n",
    "    for layer in base_model.layers:\n",
    "        layer.trainable = True\n",
    "    for layer in base_model.layers[:freeze_till]:\n",
    "        layer.trainable = False\n",
    "\n",
    "    # Customized Classifier\n",
    "    y = Dropout(0.2)(base_model.output)\n",
    "    y = Dense(1, activation='sigmoid', kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Full Model: Pre-train Conv + Customized Classifier\n",
    "    model = Model(inputs=base_model.input, outputs=y, name='Transfer_Learning')\n",
    "    sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    print('Trainable: %d, Non-Trainable: %d' % get_params_count(model))\n",
    "    \n",
    "    if weights is not None:\n",
    "        model.load_weights(weights)\n",
    "    \n",
    "    # Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard.\n",
    "    log_name = '/DogVSCat-EP{epoch:02d}-LOSS{val_loss:.4f}.h5'\n",
    "    log_dir = datetime.now().strftime('transfer_model_%Y%m%d_%H%M')\n",
    "    if not os.path.exists(log_dir):\n",
    "        os.mkdir(log_dir)\n",
    "\n",
    "    es = EarlyStopping(monitor='val_loss', patience=20)\n",
    "    mc = ModelCheckpoint(log_dir + log_name, monitor='val_loss', save_best_only=True)\n",
    "    tb = TensorBoard(log_dir=log_dir)\n",
    "\n",
    "    model.fit(x=X_train, y=y_train, batch_size=16, epochs=nb_epoch, \n",
    "              validation_data=(X_val, y_val), callbacks=[es, mc, tb])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 锁定前116层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 6790433, Non-Trainable: 14073096\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/1\n",
      "20000/20000 [==============================] - 226s - loss: 0.0407 - acc: 0.9853 - val_loss: 0.0232 - val_acc: 0.9908\n"
     ]
    }
   ],
   "source": [
    "finetune(Xception, xception.preprocess_input, height, freeze_till=116, lr=0.045, nb_epoch=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 锁定前86层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 11632361, Non-Trainable: 9231168\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/1\n",
      "20000/20000 [==============================] - 288s - loss: 0.0157 - acc: 0.9951 - val_loss: 0.0193 - val_acc: 0.9930\n"
     ]
    }
   ],
   "source": [
    "log = './transfer_model_20171021_1548/DogVSCat-EP00-LOSS0.0232.h5'\n",
    "finetune(Xception, xception.preprocess_input, height, freeze_till=86, lr=0.001, nb_epoch=1, weights=log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 锁定前57层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 19702241, Non-Trainable: 1161288\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/1\n",
      "20000/20000 [==============================] - 386s - loss: 0.0127 - acc: 0.9963 - val_loss: 0.0190 - val_acc: 0.9932\n"
     ]
    }
   ],
   "source": [
    "log = './transfer_model_20171021_1601/DogVSCat-EP00-LOSS0.0193.h5'\n",
    "finetune(Xception, xception.preprocess_input, height, freeze_till=57, lr=0.0001, nb_epoch=1, weights=log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. CONV + Dropout + GAP + Dense1024 + Dropout + Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train, labels, shuffle=True, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def finetune_dense(MODEL, preprocess, height, freeze_till, lr, nb_epoch, weights=None):\n",
    "    # Preprocess: Standardization\n",
    "    x = Input(shape=(height, height, 3))\n",
    "    x = Lambda(preprocess)(x)\n",
    "\n",
    "    # Base Model: Freeze all conv layers\n",
    "    base_model = MODEL(include_top=False, input_tensor=x, weights='imagenet')\n",
    "    for layer in base_model.layers:\n",
    "        layer.trainable = True\n",
    "    for layer in base_model.layers[:freeze_till]:\n",
    "        layer.trainable = False\n",
    "\n",
    "    # Customized Classifier\n",
    "    y = GaussianDropout(0.2)(base_model.output)\n",
    "    y = GlobalAveragePooling2D()(y)\n",
    "    y = Dense(1024, activation='selu', kernel_initializer='he_normal')(y)\n",
    "    y = GaussianDropout(0.2)(y)\n",
    "    y = Dense(1, activation='sigmoid', kernel_initializer='he_normal')(y)\n",
    "\n",
    "    # Full Model: Pre-train Conv + Customized Classifier\n",
    "    model = Model(inputs=base_model.input, outputs=y, name='Transfer_Learning')\n",
    "    sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])\n",
    "    print('Trainable: %d, Non-Trainable: %d' % get_params_count(model))\n",
    "    \n",
    "    if weights is not None:\n",
    "        model.load_weights(weights)\n",
    "    \n",
    "    # Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard.\n",
    "    log_name = '/DogVSCat-EP{epoch:02d}-LOSS{val_loss:.4f}.h5'\n",
    "    log_dir = datetime.now().strftime('transfer_model_%Y%m%d_%H%M')\n",
    "    if not os.path.exists(log_dir):\n",
    "        os.mkdir(log_dir)\n",
    "\n",
    "    es = EarlyStopping(monitor='val_loss', patience=20)\n",
    "    mc = ModelCheckpoint(log_dir + log_name, monitor='val_loss', save_best_only=True)\n",
    "    tb = TensorBoard(log_dir=log_dir)\n",
    "\n",
    "    model.fit(x=X_train, y=y_train, batch_size=16, epochs=nb_epoch, \n",
    "              validation_data=(X_val, y_val), callbacks=[es, mc, tb])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 锁定前116层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 8887585, Non-Trainable: 14073096\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/1\n",
      "20000/20000 [==============================] - 234s - loss: 0.0485 - acc: 0.9833 - val_loss: 0.0327 - val_acc: 0.9880\n"
     ]
    }
   ],
   "source": [
    "finetune_dense(Xception, xception.preprocess_input, height, freeze_till=116, lr=0.045, nb_epoch=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 锁定前86层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 13729513, Non-Trainable: 9231168\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/1\n",
      "20000/20000 [==============================] - 293s - loss: 0.0172 - acc: 0.9949 - val_loss: 0.0229 - val_acc: 0.9916\n"
     ]
    }
   ],
   "source": [
    "log = './transfer_model_20171021_1622/DogVSCat-EP00-LOSS0.0327.h5'\n",
    "finetune_dense(Xception, xception.preprocess_input, height, freeze_till=86, lr=0.001, nb_epoch=1, weights=log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 锁定前57层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 18571441, Non-Trainable: 4389240\n",
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/1\n",
      "20000/20000 [==============================] - 352s - loss: 0.0110 - acc: 0.9966 - val_loss: 0.0225 - val_acc: 0.9920\n"
     ]
    }
   ],
   "source": [
    "log = './transfer_model_20171021_1628/DogVSCat-EP00-LOSS0.0229.h5'\n",
    "finetune_dense(Xception, xception.preprocess_input, height, freeze_till=57, lr=0.0001, nb_epoch=1, weights=log)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solution 2: Extract Feature Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def export_gap(MODEL, preprocess=None, batch_size=128):\n",
    "    x = Input(shape=(height, height, 3))\n",
    "    x = Lambda(preprocess)(x)\n",
    "    model = MODEL(include_top=False, input_tensor=x, weights='imagenet', pooling='avg')\n",
    "    train_gap = model.predict(train, batch_size=batch_size)\n",
    "    test_gap = model.predict(test, batch_size=batch_size)\n",
    "    with h5py.File(\"gap_%s.h5\" % MODEL.__name__, 'w') as f:\n",
    "        f.create_dataset('train', data=train_gap)\n",
    "        f.create_dataset('test', data=test_gap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "export_gap(InceptionV3, inception_v3.preprocess_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "export_gap(Xception, xception.preprocess_input, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_temp = []\n",
    "test_temp = []\n",
    "# 'gap_InceptionV3.h5', 'gap_Xception.h5'\n",
    "for gapfile in ['gap_Xception.h5']:\n",
    "    with h5py.File(gapfile, 'r') as f:\n",
    "        train_temp.append(np.array(f['train']))\n",
    "        test_temp.append(np.array(f['test']))\n",
    "train_gap = np.concatenate(train_temp, axis=1)\n",
    "test_gap = np.concatenate(test_temp, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train_gap, labels, shuffle=True, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trainable: 2049, Non-Trainable: 0\n"
     ]
    }
   ],
   "source": [
    "x = Input(shape=(X_train.shape[1],))\n",
    "y = Dropout(0.2)(x)\n",
    "y = Dense(1, activation='sigmoid', kernel_initializer='he_normal', name='classifier')(y)\n",
    "model_gap = Model(inputs=x, outputs=y, name='GAP')\n",
    "model_gap.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy'])\n",
    "print('Trainable: %d, Non-Trainable: %d' % get_params_count(model_gap))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 20000 samples, validate on 5000 samples\n",
      "Epoch 1/5\n",
      "20000/20000 [==============================] - 4s - loss: 0.0870 - acc: 0.9871 - val_loss: 0.0326 - val_acc: 0.9910\n",
      "Epoch 2/5\n",
      "20000/20000 [==============================] - 3s - loss: 0.0255 - acc: 0.9934 - val_loss: 0.0265 - val_acc: 0.9922\n",
      "Epoch 3/5\n",
      "20000/20000 [==============================] - 4s - loss: 0.0215 - acc: 0.9939 - val_loss: 0.0244 - val_acc: 0.9922\n",
      "Epoch 4/5\n",
      "20000/20000 [==============================] - 3s - loss: 0.0196 - acc: 0.9940 - val_loss: 0.0236 - val_acc: 0.9922\n",
      "Epoch 5/5\n",
      "20000/20000 [==============================] - 3s - loss: 0.0186 - acc: 0.9943 - val_loss: 0.0232 - val_acc: 0.9920\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x260e862ec88>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare Callbacks for Model Checkpoint, Early Stopping and Tensorboard.\n",
    "log_name = '/DogVSCat-EP{epoch:02d}-LOSS{val_loss:.4f}.h5'\n",
    "log_dir = datetime.now().strftime('gap_model_%Y%m%d_%H%M')\n",
    "if not os.path.exists(log_dir):\n",
    "    os.mkdir(log_dir)\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', patience=20)\n",
    "mc = ModelCheckpoint(log_dir + log_name, monitor='val_loss', save_best_only=True)\n",
    "tb = TensorBoard(log_dir=log_dir)\n",
    "\n",
    "model_gap.fit(x=X_train, y=y_train, batch_size=16, epochs=5, validation_data=(X_val, y_val), callbacks=[es, mc, tb])\n",
    "\n",
    "# Use all training data.\n",
    "# model_gap.fit(x=train_gap, y=labels, batch_size=16, shuffle=True, epochs=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = model_gap.predict(test_gap)\n",
    "y_pred = y_pred.clip(min=0.005, max=0.995)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('test.csv', 'w') as f:\n",
    "    f.writelines('id,label\\n')\n",
    "    for i in range(12500):\n",
    "        f.writelines(str(i+1) + ',' + str(y_pred[i][0]) + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 结论\n",
    "\n",
    "- 正确的使用Finetune，的确可以达到比特征向量法更好的效果（0.0190 vs 0.0232）。关键在于掌控好学习率随解锁层数的动态变化。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
